package cn.cloud9;

import cn.cloud9.fix.DataFixApplication;
import cn.cloud9.po.RegionCell;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.db.Db;
import cn.hutool.db.Entity;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import cn.hutool.http.HttpUtil;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.net.HttpCookie;
import java.sql.SQLException;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.ConcurrentHashMap;

/**
 * @author OnCloud9
 * @description
 * @project RegionReptile
 * @date 2022年06月29日 20:05
 */

public class MainApplication {

    private static String YEAR;
    private static final int TIMEOUT = 3000;
    public static final int DELAY = 500;
    public static String HEADER_URL;
    public static final String COOKIE_KEY = "SF_cookie_1";
    public static final String COOKIE_HEADER_KEY = "Cookie";
    public static final String USER_AGENT = "User-Agent";
    public static final String[] BROWSER_AGENTS = {
            "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Mobile Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0",
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Safari/537.36 Edg/103.0.1264.37",
            "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"
    };
    public final static Random R = new Random();
    public static String TABLE_NAME;
    public static final String CODE = "CODE";
    public static final String PARENT_CODE = "PARENT_CODE";
    public static final String NAME = "NAME";
    public static final String LEVEL = "LEVEL";
    public static final String TYPE_CODE = "TYPE_CODE";
    public static final String LINK = "LINK";
    public static final String GEN_TIME = "GEN_TIME";
    public static final String REFERER = "Referer";

    public static Map<String, String> getNewRequestHeader() {
        return new ConcurrentHashMap<String, String>(){{
            this.put("Host", "www.stats.gov.cn");
            this.put("Upgrade-Insecure-Requests", "1");
        }};
    }

    private static HttpResponse retryConn(HttpRequest httpRequest) {
        HttpResponse httpResponse = null;
        try {
            httpResponse = httpRequest
                    .timeout(TIMEOUT)
                    .setConnectionTimeout(TIMEOUT)
                    .setReadTimeout(TIMEOUT)
                    .execute();
        } catch (Exception exception) {
            exception.printStackTrace();
            return retryConn(httpRequest);
        }
        return httpResponse;
    }


    public static void main(String[] args) throws SQLException {
        final boolean equals = "-fix".equals(args[1]);
        // 参数初始化
        if (equals) DataFixApplication.fixData(args[0]);
        else writeDate(args[0]);
    }

    private static void writeDate(String year) {
        YEAR = year;
        HEADER_URL = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/" + YEAR + "/";
        TABLE_NAME = "region" + YEAR;

        Db db = Db.use();
        try {
            db.execute("CREATE TABLE IF NOT EXISTS `"+ TABLE_NAME +"` (\n" +
                    "  `CODE` varchar(24) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '行政区代码',\n" +
                    "  `PARENT_CODE` varchar(24) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '上级代码',\n" +
                    "  `NAME` varchar(24) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '名称',\n" +
                    "  `LINK` varchar(252) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT '网页地址',\n" +
                    "  `LEVEL` int DEFAULT NULL COMMENT '层级',\n" +
                    "  `TYPE_CODE` varchar(12) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '类型',\n" +
                    "  `GEN_TIME` datetime DEFAULT NULL COMMENT '创建时间',\n" +
                    "  PRIMARY KEY (`CODE`) USING BTREE, \n" +
                    "  KEY `IDX_LEVEL` (`LEVEL`) USING BTREE,\n" +
                    "  KEY `IDX_PC` (`PARENT_CODE`) USING BTREE \n" +
                    ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci ROW_FORMAT=DYNAMIC;", null);

            // 清空表记录
//            int execute = db.execute("TRUNCATE TABLE " + TABLE_NAME, null);
        } catch (SQLException e) {
            e.printStackTrace();
        }



        // 第一层请求
        HttpRequest getRequest = HttpUtil.createGet(HEADER_URL);

        HttpResponse httpResponse = getRequest.execute();

        HttpCookie cookie = httpResponse.getCookie(COOKIE_KEY);
        System.out.println(cookie);

        // 取响应状态
        int status = httpResponse.getStatus();
        // 取响应data
        String body = httpResponse.body();
        System.out.println("省 响应状态 " + status);
        System.out.println("省 响应状态 " + body);
        if (status != 200) {
            System.out.println("爬取异常，程序终止");
            return;
        }

        final Document DOC = Jsoup.parse(body);
        Elements provinceTrList = DOC.getElementsByClass("provincetr");
        provinceTrList.forEach(tr -> {
            Elements provinceAList = tr.getElementsByTag("a");
            provinceAList.forEach(a -> {
                int provinceTrIndex = provinceTrList.indexOf(tr) + 1;
                int provinceALinkIndex = provinceAList.indexOf(a) + 1;

                RegionCell cell = RegionCell.builder()
                        .name(a.text())
                        .code(String.valueOf(provinceTrIndex) + provinceALinkIndex)
                        .link(HEADER_URL + a.attr("href"))
                        .parentCode(String.valueOf(0))
                        .genTime(LocalDateTime.now())
                        .build();
                System.out.println(cell);

                // 插入省份
                try {
                    db.insertOrUpdate(Entity
                            .create(TABLE_NAME)
                            .set(CODE, cell.getCode())
                            .set(PARENT_CODE, cell.getParentCode())
                            .set(NAME, cell.getName())
                            .set(LEVEL, 1)
                            .set(LINK, cell.getLink())
                            .set(GEN_TIME, cell.getGenTime())
                    );
                } catch (SQLException e) {
                    e.printStackTrace();
                }

                // 请求之前缓一缓

                // 设置header头
                Map<String, String> header = getNewRequestHeader();
                header.put(COOKIE_HEADER_KEY, cookie.toString());
                header.put(USER_AGENT, BROWSER_AGENTS[R.nextInt(BROWSER_AGENTS.length)]);
                header.put(REFERER,  HEADER_URL);

                HttpResponse response = null;
                try {
                    response = HttpUtil.createGet(cell.getLink()).addHeaders(header).execute();
                } catch (Exception exception) {
                    exception.printStackTrace();
                }


                List<HttpCookie> provinceCookies = response.getCookies();
                int provinceStatus = response.getStatus();
                System.out.println(cell.getName() + " | " + cell.getLink() + " | 请求状态" + provinceStatus);
                String provinceBody = response.body();
                final Document provinceDOC = Jsoup.parse(provinceBody);

                Elements cityTrList = provinceDOC.getElementsByClass("citytr");
                cityTrList.forEach(cityTr -> {

                    Elements cityTds = cityTr.getElementsByTag("td");
                    Element codeTd = cityTds.get(0);
                    Element nameTd = cityTds.get(1);
                    Elements asTagInCodeTd = codeTd.getElementsByTag("a");
                    Elements asTagInNameTd = nameTd.getElementsByTag("a");

                    // 取城乡代码
                    String regionCityCode;
                    String cityNextHref = null;

                    boolean isLinkTag = !CollUtil.isEmpty(asTagInCodeTd);
                    if (isLinkTag) {
                        regionCityCode = asTagInCodeTd.get(0).text();
                        cityNextHref = asTagInCodeTd.get(0).attr("href");
                    } else regionCityCode = codeTd.text();
                    String regionName;
                    if (!CollUtil.isEmpty(asTagInNameTd)) regionName = asTagInNameTd.get(0).text();
                    else regionName = nameTd.text();

                    RegionCell.RegionCellBuilder cellBuilder = RegionCell.builder();
                    if (isLinkTag)
                        cellBuilder.link(HEADER_URL + cityNextHref);

                    cellBuilder.name(regionName);
                    cellBuilder.code(regionCityCode);
                    cellBuilder.parentCode(cell.getCode());
                    cellBuilder.genTime(LocalDateTime.now()).build();
                    RegionCell cityCell = cellBuilder.build();
                    System.out.println(cityCell);

                    try {
                        db.insertOrUpdate(Entity
                                .create(TABLE_NAME)
                                .set(CODE, cityCell.getCode())
                                .set(PARENT_CODE, cityCell.getParentCode())
                                .set(NAME, cityCell.getName())
                                .set(LEVEL, 2)
                                .set(LINK, cityCell.getLink())
                                .set(GEN_TIME, cityCell.getGenTime())
                        );
                    } catch (SQLException e) {
                        e.printStackTrace();
                    }

                    if (null == cityNextHref)
                        return;

                    Map<String, String> countyHeader = getNewRequestHeader();
                    provinceCookies.forEach( provinceCookie -> countyHeader.put(COOKIE_HEADER_KEY, provinceCookie.toString() + ";"));
                    countyHeader.put(USER_AGENT, BROWSER_AGENTS[R.nextInt(BROWSER_AGENTS.length)]);
                    countyHeader.put(REFERER,  cell.getLink());
                    // HttpResponse countyResponse = HttpUtil.createGet(cityCell.getUrl()).addHeaders(countyHeader).execute();
                    HttpResponse countyResponse = retryConn(HttpUtil.createGet(cityCell.getLink()).addHeaders(countyHeader));
                    List<HttpCookie> countyCookies = countyResponse.getCookies();
                    int countyStatus = countyResponse.getStatus();
                    String countyBody = countyResponse.body();
                    System.out.println(cityCell.getName() + " | " + cityCell.getLink() + " | 请求状态" + countyStatus);
                    Document countyDOC = Jsoup.parse(countyBody);

                    Elements countyTrList = countyDOC.getElementsByClass("countytr");
                    countyTrList.forEach(countyTr -> {

                        Elements countyTds = countyTr.getElementsByTag("td");
                        Element countyCodeTd = countyTds.get(0);
                        Element countyNameTd = countyTds.get(1);
                        Elements aTagsInCountyCodeTd = countyCodeTd.getElementsByTag("a");
                        Elements aTagsInCountyNameTd = countyNameTd.getElementsByTag("a");
                        // 取城乡代码
                        String regionCountyCode;
                        String countyNextHref = null;

                        boolean isCountyLinkTag =  !CollUtil.isEmpty(aTagsInCountyCodeTd);
                        if (isCountyLinkTag) {
                            regionCountyCode = aTagsInCountyCodeTd.get(0).text();
                            countyNextHref = aTagsInCountyCodeTd.get(0).attr("href");
                        } else
                            regionCountyCode = countyCodeTd.text();
                        String regionCountyName;
                        if (!CollUtil.isEmpty(aTagsInCountyNameTd))
                            regionCountyName = aTagsInCountyNameTd.get(0).text();
                        else
                            regionCountyName = countyNameTd.text();

                        RegionCell.RegionCellBuilder countyCellBuilder = RegionCell.builder();
                        if (isCountyLinkTag) {
                            int index = countyNextHref.indexOf('/');
                            String provincePath = countyNextHref.substring(index + 1, index + 3) + "/";
                            String countyUrl = HEADER_URL + provincePath + countyNextHref;
                            countyCellBuilder.link(countyUrl);
                        }

                        countyCellBuilder.name(regionCountyName);
                        countyCellBuilder.code(regionCountyCode);
                        countyCellBuilder.genTime(LocalDateTime.now());
                        countyCellBuilder.parentCode(cityCell.getCode());
                        RegionCell countyCell = countyCellBuilder.build();
                        System.out.println(countyCell);


                        try {
                            db.insertOrUpdate(Entity
                                    .create(TABLE_NAME)
                                    .set(CODE, countyCell.getCode())
                                    .set(PARENT_CODE, countyCell.getParentCode())
                                    .set(NAME, countyCell.getName())
                                    .set(LEVEL, 3)
                                    .set(LINK, countyCell.getLink())
                                    .set(GEN_TIME, countyCell.getGenTime())
                            );
                        } catch (SQLException e) {
                            e.printStackTrace();
                        }

                        if (null == countyNextHref)
                            return;

                        Map<String, String> townHeader = getNewRequestHeader();
                        countyCookies.forEach(
                                provinceCookie -> townHeader.put(COOKIE_HEADER_KEY, provinceCookie.toString() + ";"));
                        townHeader.put(USER_AGENT, BROWSER_AGENTS[R.nextInt(BROWSER_AGENTS.length)]);
                        townHeader.put(REFERER,  countyCell.getLink());
                        // HttpResponse townResponse = HttpUtil.createGet(countyCell.getUrl()).addHeaders(townHeader).execute();
                        HttpResponse townResponse = retryConn(HttpUtil.createGet(countyCell.getLink()).addHeaders(townHeader));
                        List<HttpCookie> townCookies = townResponse.getCookies();
                        int townStatus = townResponse.getStatus();
                        String townBody = townResponse.body();
                        System.out.println(countyCell.getName() + " | " + countyCell.getLink() + " | 请求状态" + townStatus);
                        Document townDOC = Jsoup.parse(townBody);
                        Elements townTrList = townDOC.getElementsByClass("towntr");
                        townTrList.forEach(townTr -> {

                            Elements townTds = townTr.getElementsByTag("td");
                            Element townCodeTd = townTds.get(0);
                            Element townNameTd = townTds.get(1);
                            Elements aTagsInTownCodeTd = townCodeTd.getElementsByTag("a");
                            Elements aTagsInTownNameTd = townNameTd.getElementsByTag("a");
                            // 取城乡代码
                            String regionTownCode;
                            String townNextHref = null;

                            boolean isTownLinkTag = !CollUtil.isEmpty(aTagsInTownCodeTd);
                            if (isTownLinkTag) {
                                regionTownCode = aTagsInTownCodeTd.get(0).text();
                                townNextHref = aTagsInTownCodeTd.get(0).attr("href");
                            } else
                                regionTownCode = townCodeTd.text();
                            String regionTownName;
                            if (!CollUtil.isEmpty(aTagsInTownNameTd)) regionTownName = aTagsInTownNameTd.get(0).text();
                            else regionTownName = townNameTd.text();

                            final String codePrefix = regionTownCode.substring(0, 2);
                            boolean condition1 = "44".equals(codePrefix) || "46".equals(codePrefix);

                            RegionCell.RegionCellBuilder townCellBuilder = RegionCell.builder();
                            if (condition1 && isTownLinkTag) {
                                final String link = countyCell.getLink();
                                final String basePath = link.substring(0, link.lastIndexOf("/") + 1);
                                String url = basePath + townNextHref;
                                townCellBuilder.link(url);
                            } else if (isTownLinkTag) {
                                int index = townNextHref.indexOf("/");
                                String provincePath = townNextHref.substring(index + 1, index + 3);
                                String cityPath = "/" + townNextHref.substring(index + 3, index + 5) + "/";
                                String url = HEADER_URL + provincePath + cityPath + townNextHref;
                                townCellBuilder.link(url);
                            }

                            townCellBuilder.name(regionTownName);
                            townCellBuilder.code(regionTownCode);
                            townCellBuilder.genTime(LocalDateTime.now());
                            townCellBuilder.parentCode(countyCell.getCode());
                            RegionCell townCell = townCellBuilder.build();
                            System.out.println(townCell);

                            try {
                                db.insertOrUpdate(Entity
                                        .create(TABLE_NAME)
                                        .set(CODE, townCell.getCode())
                                        .set(PARENT_CODE, townCell.getParentCode())
                                        .set(NAME, townCell.getName())
                                        .set(LEVEL, 4)
                                        .set(LINK, townCell.getLink())
                                        .set(GEN_TIME, townCell.getGenTime())
                                );
                            } catch (SQLException e) {
                                e.printStackTrace();
                            }


                            if (null == townNextHref)
                                return;

                            Map<String, String> villageHeader = getNewRequestHeader();
                            townCookies.forEach( townCookie -> villageHeader.put(COOKIE_HEADER_KEY, townCookie.toString() + ";"));
                            villageHeader.put(USER_AGENT, BROWSER_AGENTS[R.nextInt(BROWSER_AGENTS.length)]);
                            villageHeader.put(REFERER,  countyCell.getLink());
                            // HttpResponse villageResponse = HttpUtil.createGet(townCell.getUrl()).addHeaders(villageHeader).execute();
                            HttpResponse villageResponse = retryConn(HttpUtil.createGet(townCell.getLink()).addHeaders(villageHeader));
                            int villageStatus = villageResponse.getStatus();
                            String villageBody = villageResponse.body();
                            System.out.println(townCell.getName() + " | " + townCell.getLink() + " | 请求状态" + villageStatus);
                            Document villageDOC = Jsoup.parse(villageBody);
                            Elements villageTrList = villageDOC.getElementsByClass("villagetr");

                            villageTrList.forEach(villageTr -> {
                                Elements villageTdList = villageTr.getElementsByTag("td");

                                Element villageCodeTd = villageTdList.get(0);
                                Element villageTypeCodeTd = villageTdList.get(1);
                                Element villageNameTd = villageTdList.get(2);

                                RegionCell.RegionCellBuilder villageCellBuilder = RegionCell.builder();
                                villageCellBuilder.code(villageCodeTd.text());
                                villageCellBuilder.name(villageNameTd.text());
                                villageCellBuilder.typeCode(villageTypeCodeTd.text());
                                villageCellBuilder.parentCode(townCell.getCode());
                                villageCellBuilder.genTime(LocalDateTime.now());
                                RegionCell villageCell = villageCellBuilder.build();
                                System.out.println(villageCell);

                                try {
                                    db.insertOrUpdate(Entity
                                            .create(TABLE_NAME)
                                            .set(CODE, villageCell.getCode())
                                            .set(PARENT_CODE, villageCell.getParentCode())
                                            .set(NAME, villageCell.getName())
                                            .set(LEVEL, 5)
                                            .set(TYPE_CODE, villageCell.getTypeCode())
                                            .set(LINK, villageCell.getLink())
                                            .set(GEN_TIME, villageCell.getGenTime())
                                    );
                                } catch (SQLException e) {
                                    e.printStackTrace();
                                }
                            });
                        });
                    });
                });
            });
        });
    }
}
